/*
 * Copyright (C) 2006-2010 Alfresco Software Limited.
 *
 * This file is part of Alfresco
 *
 * Alfresco is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Lesser General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * Alfresco is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public License
 * along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
 */

package org.filesys.server.filesys.db;

import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.List;
import java.util.jar.JarEntry;
import java.util.jar.JarFile;
import java.util.jar.JarOutputStream;

import org.filesys.debug.Debug;
import org.filesys.server.SrvSession;
import org.filesys.server.core.DeviceContext;
import org.filesys.server.filesys.DiskDeviceContext;
import org.filesys.server.filesys.FileName;
import org.filesys.server.filesys.FileOpenParams;
import org.filesys.server.filesys.FileStatus;
import org.filesys.server.filesys.NetworkFile;
import org.filesys.server.filesys.cache.FileState;
import org.filesys.server.filesys.cache.FileStateCache;
import org.filesys.server.filesys.cache.FileStateListener;
import org.filesys.server.filesys.cache.FileStateProxy;
import org.filesys.server.filesys.loader.BackgroundFileLoader;
import org.filesys.server.filesys.loader.CachedFileInfo;
import org.filesys.server.filesys.loader.FileLoader;
import org.filesys.server.filesys.loader.FileLoaderException;
import org.filesys.server.filesys.loader.FileProcessor;
import org.filesys.server.filesys.loader.FileProcessorList;
import org.filesys.server.filesys.loader.FileRequest;
import org.filesys.server.filesys.loader.FileRequestQueue;
import org.filesys.server.filesys.loader.FileSegment;
import org.filesys.server.filesys.loader.FileSegmentInfo;
import org.filesys.server.filesys.loader.MultipleFileRequest;
import org.filesys.server.filesys.loader.SingleFileRequest;
import org.filesys.util.MemorySize;
import org.springframework.extensions.config.ConfigElement;

/**
 * Database File Data Loader Class
 *
 * <p>
 * The database file data loader loads/saves file data to a BLOB field in a seperate database table
 * to the main filesystem structure. The file data is indexed using the file id generated by the
 * main database disk driver.
 *
 * <p>
 * The file data may be split up into several BLOB fields.
 *
 * <p>
 * This class relies on a seperate DBDataInterface implementation to provide the methods to load and
 * save the file data to the database table.
 *
 * @author gkspencer
 */
public class DBFileLoader implements FileLoader, BackgroundFileLoader, FileStateListener {

    // Status codes returned from the load/save worker thread processing
    public final static int StsSuccess  = 0;
    public final static int StsRequeue  = 1;
    public final static int StsError    = 2;

    // Temporary sub-directory/file/Jar prefix
    public static final String TempDirPrefix    = "ldr";
    public static final String TempFilePrefix   = "ldr_";
    public static final String JarFilePrefix    = "jar_";

    // Maximum files per temporary sub-directory
    private static final int MaximumFilesPerSubDir = 500;

    // Attributes attached to the file state
    public static final String DBFileSegmentInfo = "DBFileSegmentInfo";

    // Default/minimum/maximum number of worker threads to use
    public static final int DefaultWorkerThreads = 4;
    public static final int MinimumWorkerThreads = 1;
    public static final int MaximumWorkerThreads = 50;

    // Default/minimum files per jar and jar size settings
    public static final int DefaultFilesPerJar = 25;
    public static final int MinimumFilesPerJar = 5;

    public static final int DefaultSizePerJar = 200000;
    public static final int MinimumSizePerJar = 100000;

    // File state timeout values
    public static final long SequentialFileExpire   = 3000L;    // milliseconds
    public static final long RequestProcessedExpire = 3000L;    // "
    public static final long RequestQueuedExpire    = 10000L;   // "

    // Transaction minimum file size
    public static final int TransactionMinimumFileSize = 1024;

    // Transaction timeout default, minimum and maximum values
    public static final long DefaultTransactionTimeout = 5000L;    // milliseconds
    public static final long MinimumTransactionTimeout = 2000L;    // "
    public static final long MaximumTransactionTimeout = 60000L;   // "

    // Default file data fragment size
    public final static long DEFAULT_FRAGSIZE   = 512L * 1024L;         // 1/2Mb
    public final static long MIN_FRAGSIZE       = 64L * 1024L;          // 64Kb
    public final static long MAX_FRAGSIZE       = 1024L * 1024L * 1024L;// 1Gb

    // Memory buffer maximum size
    public final static long MAX_MEMORYBUFFER = 512L * 1024L; // 1/2Mb

    // Prefix for Jar files when added to the file state cache. The files must not be accessible
    // from the client so we use invalid file name characters to prefix the name.
    public static final String JarStatePrefix = "**JAR";

    // Default Jar file cache timeout
    public static final long JarStateTimeout = 300000L; // 5 minutes

    // Default Jar compression level

    public static final int JarDefaultCompression = 0; // no compression

    // Name, used to prefix worker thread names
    private String m_name;

    // Maximum in-memory file request size and low water mark
    private int m_maxQueueSize;
    private int m_lowQueueSize;

    // Enable debug output, additional thread level debug output
    private boolean m_debug;
    private boolean m_threadDebug;

    // Number of worker threads to create for read/write requests
    private int m_readWorkers;
    private int m_writeWorkers;

    // Small file threshold, files below this threshold will be queued to the transaction queue
    // to be bundled together into a multiple file request when there are 'n' files in the
    // transaction and/or the total file size is greater than the maximum size threshold.
    //
    // A value of zero indicates that transaction queueing is disabled.
    private long m_smallFileSize;
    private int m_filesPerJar;
    private int m_sizePerJar;

    // Database device context
    private DBDeviceContext m_dbCtx;

    // File state cache, from device context
    private FileStateCache m_stateCache;

    // Database data interface used to load/save the file and Jar file data
    private DBDataInterface m_dbDataInterface;

    // Worker thread pool for loading/saving file data
    private BackgroundLoadSave m_backgroundLoader;

    // File data fragment size
    private long m_fragSize = DEFAULT_FRAGSIZE;

    // Keep Jar files created for multiple file transaction requests
    private boolean m_keepJars;

    // Jar file compression level, 0 = no compression, 9 = highest compression
    private int m_jarCompressLevel;

    // Temporary file area
    private String m_tempDirName;
    private File m_tempDir;

    // Temporary directory/file prefixes
    private String m_tempDirPrefix = TempDirPrefix;
    private String m_tempFilePrefix = TempFilePrefix;

    // Current temporary sub-directory
    private String m_curTempName;
    private File m_curTempDir;
    private int m_curTempIdx;

    // Maximum/current number of files in a temporary directory
    private int m_tempCount;
    private int m_tempMax;

    // Current transaction id, cumulative file size, file count and time last file was added to the
    // current transaction.
    // Transaction lock used to synchronize access to the values.
    private int m_tranId;
    private int m_totFileSize;
    private int m_totFiles;
    private long m_lastTranFile;

    private Object m_tranLock = new Object();

    // Time to wait for more files to be added to a transaction before it is sent to be processed,
    // in milliseconds and
    // transaction timer thread
    private long m_tranTimeout;
    private TransactionTimer m_transTimer;

    // List of file processors that process cached files before storing and after loading.
    private FileProcessorList m_fileProcessors;

    /**
     * Transaction Timer Thread Inner Class
     */
    protected class TransactionTimer implements Runnable {

        // Transaction timer thread
        private Thread m_thread;

        // Transaction timeout and thread wakeup interval
        private long m_timeout;
        private long m_wakeup;

        // Shutdown flag
        private boolean m_shutdown = false;

        /**
         * Class constructor
         *
         * @param name    String
         * @param timeout long
         */
        public TransactionTimer(String name, long timeout) {

            // Set the transaction timeout and thread wakeup interval
            m_timeout = timeout;
            m_wakeup = timeout / 2;

            // Create the thread and start it
            m_thread = new Thread(this);
            m_thread.setName(name);
            m_thread.setDaemon(true);
            m_thread.start();
        }

        /**
         * Request the worker thread to shutdown
         */
        public final void shutdownRequest() {
            m_shutdown = true;
            try {
                m_thread.interrupt();
            }
            catch (Exception ex) {
            }
        }

        /**
         * Run the thread
         */
        public void run() {

            // Loop until shutdown
            while (m_shutdown == false) {

                try {
                    Thread.sleep(m_wakeup);
                }
                catch (InterruptedException ex) {
                }

                // Check if a shutdown has been requested
                if (m_shutdown)
                    break;

                // Check if the current transaction should be flushed to the processing queue
                if (m_lastTranFile != 0L) {

                    // Get the current time
                    long timeNow = System.currentTimeMillis();

                    synchronized (m_tranLock) {

                        // Check if the transaction has timed out
                        if ((m_lastTranFile + m_timeout) < timeNow) {

                            // Wakeup the transaction loader to send the current transaction
                            m_backgroundLoader.flushTransaction(m_tranId);

                            // Update the current transaction details
                            m_tranId++;
                            m_totFiles = 0;
                            m_totFileSize = 0;
                            m_lastTranFile = 0L;

                            // DEBUG
                            if (Debug.EnableInfo && hasDebug())
                                Debug.println("BackgroundLoadSave Transaction timed out, queued for loading");
                        }
                    }
                }
            }

            // DEBUG
            if (Debug.EnableInfo && hasDebug())
                Debug.println("BackgroundLoadSave Transaction timer shutdown");
        }
    }

    ;

    /**
     * Class constructor
     */
    public DBFileLoader() {
    }

    /**
     * Return the database features required by this file loader. Return zero if no database
     * features are required by the loader.
     *
     * @return int
     */
    public int getRequiredDBFeatures() {

        // Return the database features required by the loader
        return DBInterface.FeatureData + DBInterface.FeatureJarData + DBInterface.FeatureQueue;
    }

    /**
     * Return the database device context
     *
     * @return DBDeviceContext
     */
    public final DBDeviceContext getContext() {
        return m_dbCtx;
    }

    /**
     * Return the Jar compression level
     *
     * @return int
     */
    public final int getJarCompressionLevel() {
        return m_jarCompressLevel;
    }

    /**
     * Return the file state cache
     *
     * @return FileStateCache
     */
    protected final FileStateCache getStateCache() {
        return m_stateCache;
    }

    /**
     * Return the temporary directory name
     *
     * @return String
     */
    public final String getTemporaryDirectoryPath() {
        return m_tempDirName;
    }

    /**
     * Return the temporary directory
     *
     * @return File
     */
    public final File getTemporaryDirectory() {
        return m_tempDir;
    }

    /**
     * Return the current temporry sub-directory
     *
     * @return File
     */
    public final File getCurrentTempDirectory() {
        return m_curTempDir;
    }

    /**
     * Check if Jars files should be kept in the temporary area
     *
     * @return boolean
     */
    public final boolean hasKeepJars() {
        return m_keepJars;
    }

    /**
     * Return the database data interface
     *
     * @return DBDataInterface
     */
    public final DBDataInterface getDBDataInterface() {
        return m_dbDataInterface;
    }

    /**
     * Add a file processor to process files before storing and after loading.
     *
     * @param fileProc FileProcessor
     * @throws FileLoaderException Error adding the file processor
     */
    public void addFileProcessor(FileProcessor fileProc)
            throws FileLoaderException {

        // Check if the file processor list has been allocated
        if (m_fileProcessors == null)
            m_fileProcessors = new FileProcessorList();

        // Add the file processor
        m_fileProcessors.addProcessor(fileProc);
    }

    /**
     * Determine if there are any file processors configured
     *
     * @return boolean
     */
    public final boolean hasFileProcessors() {
        return m_fileProcessors != null ? true : false;
    }

    /**
     * Check if debug output is enabled
     *
     * @return boolean
     */
    public final boolean hasDebug() {
        return m_debug;
    }

    /**
     * Return the maximum in-memory file request queue size
     *
     * @return int
     */
    public final int getMaximumQueueSize() {
        return m_maxQueueSize;
    }

    /**
     * Return the in-memory file request queue low water mark level
     *
     * @return int
     */
    public final int getLowQueueSize() {
        return m_lowQueueSize;
    }

    /**
     * Return the worker thread prefix
     *
     * @return String
     */
    public final String getName() {
        return m_name;
    }

    /**
     * Get the small file threshold size
     *
     * @return long
     */
    public final long getSmallFileSize() {
        return m_smallFileSize;
    }

    /**
     * Get the number of files per Jar
     *
     * @return int
     */
    public final int getFilesPerJar() {
        return m_filesPerJar;
    }

    /**
     * Get the file size limit for packing into Jars
     *
     * @return int
     */
    public final int getJarFileSize() {
        return m_sizePerJar;
    }

    /**
     * Get the transaction timeout value, in milliseconds
     *
     * @return long
     */
    public final long getTransactionTimeout() {
        return m_tranTimeout;
    }

    /**
     * Return the temporary sub-directory prefix
     *
     * @return String
     */
    public final String getTempDirectoryPrefix() {
        return m_tempDirPrefix;
    }

    /**
     * Return the temporary file prefix
     *
     * @return String
     */
    public final String getTempFilePrefix() {
        return m_tempFilePrefix;
    }

    /**
     * Set the worker thread name prefix
     *
     * @param name String
     */
    protected final void setName(String name) {
        m_name = name;
    }

    /**
     * Create a network file for the specified file
     *
     * @param params FileOpenParams
     * @param fid    int
     * @param stid   int
     * @param did    int
     * @param create boolean
     * @param dir    boolean
     * @throws IOException I/O error
     * @throws FileNotFoundException File not found
     */
    public NetworkFile openFile(FileOpenParams params, int fid, int stid, int did, boolean create, boolean dir)
            throws IOException, FileNotFoundException {

        // Split the file name to get the name only
        String[] paths = FileName.splitPath(params.getPath());
        String name = paths[1];

        // Find, or create, the file state for the file/directory
        FileState fstate = m_stateCache.findFileState(params.getFullPath(), true);
        fstate.setExpiryTime(System.currentTimeMillis() + m_stateCache.getFileStateExpireInterval());

        // Check if the file is a directory
        DBNetworkFile netFile = null;

        if (dir == false) {

            // Create the network file and associated file segment
            CachedNetworkFile cacheFile = createNetworkFile(fstate, params, name, fid, stid, did);
            netFile = cacheFile;

            // Check if the file is being opened for sequential access and the data has not yet been
            // loaded
            FileSegment fileSeg = cacheFile.getFileSegment();

            if (create == true || params.isOverwrite() == true) {

                // Indicate that the file data is available, this is a new file or the existing file
                // is being overwritten
                // so there is no data to load.
                fileSeg.setStatus(FileSegmentInfo.Available);
            } else if (params.isSequentialAccessOnly() && fileSeg.isDataLoading() == false) {

                synchronized (cacheFile.getFileState()) {

                    // Create the temporary file
                    cacheFile.openFile(create);
                    cacheFile.closeFile();

                    // Queue a file data load request
                    if (fileSeg.isDataLoading() == false)
                        queueFileRequest(new SingleFileRequest(FileRequest.LOAD, cacheFile.getFileId(), cacheFile.getStreamId(),
                                fileSeg.getInfo(), cacheFile.getFullNameStream(), fstate));
                }

                // DEBUG
                if (Debug.EnableInfo && hasDebug())
                    Debug.println("## FileLoader Queued file load, SEQUENTIAL access");
            }
        } else {

            // Create a placeholder network file for the directory
            netFile = new DirectoryNetworkFile(name, fid, did, m_stateCache.getFileStateProxy(fstate));
            netFile.setFullName(params.getPath());

            // Debug
            if (Debug.EnableInfo && hasDebug())
                Debug.println("DBFileLoader.openFile() DIR state=" + fstate + ", netfile=" + netFile);
        }

        // Return the network file
        return netFile;
    }

    /**
     * Close the network file
     *
     * @param sess    SrvSession
     * @param netFile NetworkFile
     * @throws IOException I/O error
     */
    public void closeFile(SrvSession sess, NetworkFile netFile)
            throws IOException {

        // Close the cached network file
        if (netFile instanceof CachedNetworkFile) {

            // Get the cached network file
            CachedNetworkFile cacheFile = (CachedNetworkFile) netFile;
            cacheFile.closeFile();

            // Get the file segment details
            FileSegment fileSeg = cacheFile.getFileSegment();

            // Check if the file data has been updated, if so then queue a file save
            if (fileSeg.isUpdated() && netFile.hasDeleteOnClose() == false) {

                // Set the modified date/time and file size for the file
                File tempFile = new File(fileSeg.getTemporaryFile());

                netFile.setModifyDate(tempFile.lastModified());
                netFile.setFileSize(tempFile.length());

                // Queue a file save request to save the data back to the repository, if not already
                // queued
                if (fileSeg.isSaveQueued() == false) {

                    // Indicate that the file data has been updated
                    m_stateCache.setDataUpdateInProgress(cacheFile.getFileState());

                    // Create a file save request for the updated file segment
                    SingleFileRequest fileReq = new SingleFileRequest(FileRequest.SAVE, cacheFile.getFileId(), cacheFile.getStreamId(),
                            fileSeg.getInfo(), cacheFile.getFileState().getPath(), cacheFile.getFileState());
                    // Set the file segment status
                    fileSeg.setStatus(FileSegmentInfo.SaveWait, true);

                    // Check if the request should be part of a transaction
                    if (getSmallFileSize() > 0 && netFile.getFileSize() < getSmallFileSize()) {

                        // Make the file request into a transaction request
                        createTransactionRequest(fileReq, netFile);
                    }

                    // Queue the file save request
                    queueFileRequest(fileReq);
                } else if (Debug.EnableInfo && hasDebug()) {

                    // DEBUG
                    Debug.println("## FileLoader Save already queued for " + fileSeg);
                }
            }

            // Update the cache timeout for the temporary file if there are no references to the
            // file. If the file was
            // opened for sequential access only it will be expired quicker.
            else if (cacheFile.getFileState().getOpenCount() == 0) {

                // If the file was opened for sequential access only then we can delete it from the
                // temporary area sooner
                long tmo = System.currentTimeMillis();

                if (cacheFile.isSequentialOnly())
                    tmo += SequentialFileExpire;
                else
                    tmo += m_stateCache.getFileStateExpireInterval();

                // Set the file state expiry, the local file data will be deleted when the file
                // state expires (if there
                // are still no references to the file).
                cacheFile.getFileState().setExpiryTime(tmo);
            }
        }
    }

    /**
     * Delete the specified file data
     *
     * @param fname String
     * @param fid   int
     * @param stid  int
     * @throws IOException I/O error
     */
    public void deleteFile(String fname, int fid, int stid)
            throws IOException {

        // Delete the file data from the database
        try {

            // Find the associated file state
            FileState fstate = m_stateCache.findFileState(fname, false);

            if (fstate != null) {

                // Get the file segment details
                FileSegmentInfo fileSegInfo = (FileSegmentInfo) fstate.removeAttribute(DBFileSegmentInfo);
                if (fileSegInfo != null) {
                    try {

                        // Delete the temporary file
                        fileSegInfo.deleteTemporaryFile();
                    }
                    catch (Exception ex) {

                        // DEBUG
                        if (Debug.EnableInfo && hasDebug())
                            Debug.println("## DBFileLoader failed to delete temp file " + fileSegInfo.getTemporaryFile());
                    }
                }
            }

            // Delete the data from the database table
            getDBDataInterface().deleteFileData(fid, stid);
        }
        catch (Exception ex) {

            // DEBUG
            if (Debug.EnableInfo && hasDebug())
                Debug.println("## DBFileLoader deleteFile() error, " + ex.toString());
        }
    }

    /**
     * Request file data to be loaded/saved
     *
     * @param req FileRequest
     */
    public void queueFileRequest(FileRequest req) {

        // Pass the request to the background load/save thread pool
        m_backgroundLoader.queueFileRequest(req);
    }

    /**
     * Load a file
     *
     * @param req FileRequest
     * @return int
     * @throws Exception Error loading the file data
     */
    public int loadFile(FileRequest req)
            throws Exception {

        // DEBUG
        long startTime = 0L;
        SingleFileRequest loadReq = (SingleFileRequest) req;

        if (Debug.EnableInfo && hasDebug()) {
            Debug.println("## DBFileLoader loadFile() req=" + loadReq.toString() + ", thread="
                    + Thread.currentThread().getName());
            startTime = System.currentTimeMillis();
        }

        // Check if the temporary file still exists, if not then the file has been deleted from the
        // filesystem
        File tempFile = new File(loadReq.getTemporaryFile());
        FileSegment fileSeg = findFileSegmentForPath(loadReq.getVirtualPath());

        if (tempFile.exists() == false || fileSeg == null) {

            // DEBUG
            if (Debug.EnableInfo && hasDebug())
                Debug.println("  Temporary file deleted");

            // Return an error status
            fileSeg.setStatus(FileSegmentInfo.Error, false);
            return StsError;
        }

        // Load the file data
        FileOutputStream fileOut = null;
        int loadSts = StsRequeue;

        try {

            // Update the segment status
            fileSeg.setStatus(FileSegmentInfo.Loading);

            // Get the file data details
            DBDataDetails dataDetails = getDBDataInterface().getFileDataDetails(loadReq.getFileId(), loadReq.getStreamId());

            // DEBUG
            if (Debug.EnableInfo && hasDebug())
                Debug.println("  Data details: " + dataDetails);

            // Check if the file is packaged in a Jar
            if (dataDetails.isStoredInJar()) {

                // Load the file data from a Jar file
                loadSts = loadFileFromJar(loadReq, tempFile, dataDetails);

                // Update the file status, and clear the queued flag
                fileSeg.setStatus(FileSegmentInfo.Available, false);
                fileSeg.signalDataAvailable();
            } else {

                // Load the file data from the main file record(s)
                getDBDataInterface().loadFileData(loadReq.getFileId(), loadReq.getStreamId(), fileSeg);

                // Set the load status
                loadSts = StsSuccess;

                // DEBUG
                if (Debug.EnableInfo && hasDebug()) {
                    long endTime = System.currentTimeMillis();
                    Debug.println("## DBFileLoader loaded fid=" + loadReq.getFileId() + ", stream=" + loadReq.getStreamId()
                            + ", frags=" + dataDetails.numberOfDataFragments() + ", time=" + (endTime - startTime) + "ms");
                }
            }
        }
        catch (DBException ex) {

            // DEBUG
            if (Debug.EnableError && hasDebug())
                Debug.println(ex);

            // Indicate the file load failed
            loadSts = StsError;
        }
        catch (IOException ex) {

            // DEBUG
            if (Debug.EnableError && hasDebug())
                Debug.println(ex);

            // Indicate the file load failed
            loadSts = StsError;
        }

        // Check if the file was loaded successfully
        if (loadSts == StsSuccess) {

            // Signal that the file data is available
            fileSeg.signalDataAvailable();

            // Update the file status
            fileSeg.setStatus(FileSegmentInfo.Available, false);

            // Run the file load processors
            runFileLoadedProcessors(getContext(), loadReq.getFileState(), fileSeg);
        }

        // Return the load file status
        return loadSts;
    }

    /**
     * Load the requested file from a Jar file. The Jar file must first be loaded
     * from the database, then the file data is unpacked to the temporary file. The Jar file is
     * cached as there may be other files accessed in the same Jar file.
     *
     * @param loadReq     SingleFileRequest
     * @param tempFile    File
     * @param dataDetails DBDataDetails
     * @return int
     */
    protected final int loadFileFromJar(SingleFileRequest loadReq, File tempFile, DBDataDetails dataDetails) {

        // Check if the Jar file has already been loaded, if so there will be a file state
        int loadSts = StsError;

        String jarStateName = JarStatePrefix + dataDetails.getJarId();
        FileState jarState = getStateCache().findFileState(jarStateName);
        FileSegmentInfo segInfo = null;
        FileSegment jarSeg = null;
        File jarFile = null;

        // Check if the Jar already exists in the temporary file area, if not then load the Jar from
        // the Centera
        boolean loadJarReq = false;

        if (jarState == null) {

            // Create a new file state for the Jar file
            jarFile = new File(getCurrentTempDirectory(), JarFilePrefix + dataDetails.getJarId() + ".jar");
            jarState = createFileStateForRequest(-2, jarFile.getAbsolutePath(), jarStateName, FileSegmentInfo.LoadWait);

            jarState.setExpiryTime(System.currentTimeMillis() + JarStateTimeout);
            segInfo = new FileSegmentInfo(jarFile.getAbsolutePath());
            jarState.addAttribute(DBFileSegmentInfo, segInfo);
            jarSeg = new FileSegment(segInfo, true);

            // Indicate that the Jar requires loading
            loadJarReq = true;

            // DEBUG
            if (Debug.EnableInfo && hasDebug())
                Debug.println("Creating new state for Jar, jar=" + jarFile.getAbsolutePath());
        } else {

            // Get the file segment status
            segInfo = (FileSegmentInfo) jarState.findAttribute(DBFileSegmentInfo);
            if (segInfo != null && segInfo.hasStatus() == FileSegmentInfo.Initial) {

                // Set the Jar file
                jarFile = new File(jarSeg.getTemporaryFile());

                // Indicate that the Jar file must be loaded
                loadJarReq = true;

                // DEBUG
                if (Debug.EnableInfo && hasDebug())
                    Debug.println("Jar file state requires Jar load, state=Initial");
            } else if (jarSeg == null) {

                synchronized (jarState) {

                    // Create a new file segment
                    jarFile = new File(getCurrentTempDirectory(), dataDetails.getJarId() + ".jar");
                    segInfo = new FileSegmentInfo();
                    segInfo.setTemporaryFile(jarFile.getAbsolutePath());

                    jarSeg = new FileSegment(segInfo, true);
                    jarSeg.setStatus(FileSegmentInfo.LoadWait, true);

                    // Add the segment to the file state cache
                    jarState.addAttribute(DBFileSegmentInfo, segInfo);
                }

                // Indicate that the Jar file must be loaded
                loadJarReq = true;

                // DEBUG
                if (Debug.EnableInfo && hasDebug())
                    Debug.println("Jar file segment created, load required");
            } else {

                // DEBUG
                if (Debug.EnableInfo && hasDebug())
                    Debug.println("jarSeg=" + jarSeg);
            }
        }

        // Check if the Jar file requires loading
        if (loadJarReq == true) {

            try {

                // Get the load lock for the Jar file, if we get the lock then load the file data
                if (jarSeg.getLoadLock() == true) {

                    // DEBUG
                    if (Debug.EnableInfo && hasDebug())
                        Debug.println("Loading Jar, got load lock ...");

                    try {

                        // Load the Jar file data from the database
                        getDBDataInterface().loadJarData(dataDetails.getJarId(), jarSeg);

                        // Set the Jar file segment status to indicate that the data has been loaded
                        jarSeg.setStatus(FileSegmentInfo.Available, false);

                        // Indicate load successful
                        loadSts = StsSuccess;

                        // DEBUG
                        if (Debug.EnableInfo && hasDebug())
                            Debug.println("## DBFileLoader JAR loaded " + loadReq.toString() + ", jarId="
                                    + dataDetails.getJarId());
                    }
                    catch (Exception ex) {
                        Debug.println(ex);
                    }
                    finally {

                        // Wakeup any other threads waiting on the Jar file load
                        synchronized (jarState) {
                            jarState.notifyAll();
                        }
                    }
                } else {

                    // Check if the file data is now available
                    if (jarSeg.hasStatus() == FileSegmentInfo.Available) {

                        // Indicate that the Jar load was successful
                        loadSts = StsSuccess;

                        // DEBUG
                        if (Debug.EnableInfo && hasDebug())
                            Debug.println("Waited for load, threadId=" + loadReq.getThreadId());
                    } else
                        loadSts = StsRequeue;
                }
            }
            catch (InterruptedException ex) {
                loadSts = StsRequeue;
            }
        } else {

            // Bump the Jar file state expiry so that it stays in the cache a while longer, might
            // get more hits
            jarState.setExpiryTime(System.currentTimeMillis() + JarStateTimeout);

            // Get the Jar file segment
            segInfo = (FileSegmentInfo) jarState.findAttribute(DBFileSegmentInfo);
            jarSeg = new FileSegment(segInfo, true);

            // Check if the Jar data is available
            if (jarSeg.hasStatus() != FileSegmentInfo.Available) {

                // DEBUG
                if (Debug.EnableInfo && hasDebug())
                    Debug.println("## DBFileLoader Jar not yet available, " + jarSeg.getTemporaryFile());

                // Wait until the Jar has been loaded
                int retryCnt = 0;

                while (retryCnt++ < 50 && jarSeg.hasStatus() != FileSegmentInfo.Available) {

                    // Sleep for a while
                    try {
                        Thread.sleep(250);
                    }
                    catch (InterruptedException ex) {
                    }
                }

                // Check if the Jar file has been loaded
                if (jarSeg.hasStatus() != FileSegmentInfo.Available) {

                    // Requeue the file load request
                    loadSts = StsRequeue;
                } else {

                    // DEBUG
                    if (Debug.EnableInfo && hasDebug())
                        Debug.println("  Jar file loaded, waited " + (250 * retryCnt) + "ms");
                }
            } else {

                // Indicate that the Jar file is loaded
                loadSts = StsSuccess;

                // DEBUG
                if (Debug.EnableInfo && hasDebug())
                    Debug.println("## DBFileLoader Jar cache hit, re-using " + jarSeg.getTemporaryFile());
            }
        }

        // If the Jar file has been loaded, or is available, then extract the required file
        if (loadSts == StsSuccess) {

            // Open the Jar file and copy the required file data to the temporary file
            JarFile jar = null;
            FileOutputStream outFile = null;
            InputStream jarIn = null;

            try {

                // Open the Jar file
                jar = new JarFile(jarSeg.getTemporaryFile());

                // Find the required entry in the Jar
                String entryName = null;

                if (loadReq.getStreamId() > 0)
                    entryName = getTempFilePrefix() + loadReq.getFileId() + "_" + loadReq.getStreamId() + ".tmp";
                else
                    entryName = getTempFilePrefix() + loadReq.getFileId() + ".tmp";

                JarEntry jarEntry = jar.getJarEntry(entryName);

                if (jarEntry != null) {

                    // Open the Jar entry to read the data and temporary file to write the data to
                    jarIn = jar.getInputStream(jarEntry);
                    outFile = new FileOutputStream(tempFile);

                    // Create a buffer to read/write the file data
                    long startTime = System.currentTimeMillis();

                    byte[] buf = new byte[1024];

                    int totLen = 0;
                    int rdlen = jarIn.read(buf);

                    while (rdlen > 0) {

                        // Write the data to the temporary file
                        outFile.write(buf, 0, rdlen);
                        totLen += rdlen;

                        // Read another buffer of data from the Jar file
                        rdlen = jarIn.read(buf);
                    }

                    long stopTime = System.currentTimeMillis();

                    // DEBUG
                    if (Debug.EnableInfo && hasDebug())
                        Debug.println("Loaded file " + jarEntry.getName() + ", size=" + totLen + ", in " + (stopTime - startTime)
                                + "ms");

                    // Close the Jar stream and output file
                    jarIn.close();
                    jarIn = null;

                    outFile.close();
                    outFile = null;
                } else {

                    // DEBUG
                    if (Debug.EnableInfo && hasDebug())
                        Debug.println("## DBFileLoader Failed to find file in Jar, fid=" + loadReq.getFileId() + ", jar="
                                + jarSeg.getTemporaryFile());

                    // Set the load status to indicate error
                    loadSts = StsError;
                }
            }
            catch (Exception ex) {
                if (Debug.EnableError) {
                    Debug.println("Error in worker thread=" + loadReq.getThreadId());
                    Debug.println(ex);
                }
            }
            finally {

                // Close the Jar entry
                if (jarIn != null) {
                    try {
                        jarIn.close();
                    }
                    catch (IOException ex) {
                    }
                }

                // Close the Jar file
                if (jar != null) {
                    try {
                        jar.close();
                    }
                    catch (IOException ex) {
                    }
                }

                // Close the output file
                if (outFile != null) {
                    try {
                        outFile.close();
                    }
                    catch (IOException ex) {
                        Debug.println(ex);
                    }
                }
            }
        }

        // Return the load file status
        return loadSts;
    }

    /**
     * Store a file
     *
     * @param req FileRequest
     * @return int
     * @throws Exception Error saving the file data
     */
    public int storeFile(FileRequest req)
            throws Exception {

        // Check for a single file request
        int saveSts = StsError;

        if (req instanceof SingleFileRequest) {

            // Process a single file save request
            SingleFileRequest singleReq = (SingleFileRequest) req;

            saveSts = storeSingleFile(singleReq);

            // If the file data save was successful, or had an unrecoverable error, then update the file
            // data status
            if (saveSts == StsSuccess || saveSts == StsError)
                m_stateCache.setDataUpdateCompleted(singleReq.getFileState());
        }

        // Check for a multi file request
        else if (req instanceof MultipleFileRequest) {

            // Process a multi file save request
            MultipleFileRequest multiReq = (MultipleFileRequest) req;

            saveSts = storeMultipleFile(multiReq);

            // If the file data save was successful, or had an unrecoverable error, then update the file
            // data status for each file in the multi file save
            if (saveSts == StsSuccess || saveSts == StsError) {

                // Update the data save status for each file in the multi file save
                for (int idx = 0; idx < multiReq.getNumberOfFiles(); idx++) {

                    // Get the current cached file details
                    CachedFileInfo cacheFileInfo = multiReq.getFileInfo(idx);
                    if (cacheFileInfo != null && cacheFileInfo.hasFileState())
                        m_stateCache.setDataUpdateCompleted(cacheFileInfo.getFileState());
                }
            }
        } else {

            // Unknown request type
            if (Debug.EnableError)
                Debug.println("## DBFileLoader Unknown save type - " + req.getClass().getName());
        }

        // Return the data save status
        return saveSts;
    }

    /**
     * Process a store single file request
     *
     * @param saveReq SingleFileRequest
     * @return int
     * @throws Exception Error saving the file data
     */
    protected final int storeSingleFile(SingleFileRequest saveReq)
            throws Exception {

        // DEBUG
        long startTime = 0L;

        if (Debug.EnableInfo && hasDebug()) {
            Debug.println("## DBFileLoader storeFile() req=" + saveReq.toString() + ", thread="
                    + Thread.currentThread().getName());
            startTime = System.currentTimeMillis();
        }

        // Check if the temporary file still exists, if not then the file has been deleted from the
        // filesystem
        File tempFile = new File(saveReq.getTemporaryFile());
        FileSegment fileSeg = findFileSegmentForPath(saveReq.getVirtualPath());

        if (tempFile.exists() == false || fileSeg == null) {

            // DEBUG
            if (Debug.EnableInfo && hasDebug())
                Debug.println("  Temporary file deleted");

            // Return an error status
            return StsError;
        }

        // Run any file store processors
        runFileStoreProcessors(m_dbCtx, saveReq.getFileState(), fileSeg);

        // Get the temporary file size
        long fileSize = tempFile.length();

        // DEBUG
        if (Debug.EnableInfo && hasDebug())
            Debug.println("## DBFileLoader fileSize=" + fileSize);

        // Update the segment status, and clear the updated flag
        fileSeg.setStatus(FileSegmentInfo.Saving);
        fileSeg.getInfo().setUpdated(false);

        try {

            // Save the file data to the database
            getDBDataInterface().saveFileData(saveReq.getFileId(), saveReq.getStreamId(), fileSeg);
        }
        catch (DBException ex) {
            Debug.println(ex);
        }
        catch (IOException ex) {
            Debug.println(ex);
        }

        // DEBUG
        if (Debug.EnableInfo && hasDebug()) {
            long endTime = System.currentTimeMillis();
            Debug.println("## DBFileLoader saved file=" + saveReq.toString() + ", time=" + (endTime - startTime) + "ms");
        }

        // Update the segment status
        fileSeg.setStatus(FileSegmentInfo.Saved, false);

        // Indicate that the file save request was processed
        return StsSuccess;
    }

    /**
     * Process a store multiple file request
     *
     * @param saveReq MultipleFileRequest
     * @return int
     * @throws Exception Error saving multiple file data
     */
    protected final int storeMultipleFile(MultipleFileRequest saveReq)
            throws Exception {

        // Create the Jar file and pack all temporary files
        File jarFile = null;
        JarOutputStream outJar = null;

        try {

            // Create the Jar file in the temporary cache area
            jarFile = File.createTempFile("JAR_", ".jar", getCurrentTempDirectory());

            FileOutputStream outFile = new FileOutputStream(jarFile);
            outJar = new JarOutputStream(outFile);

            // Set the Jar compression level (0=no compression, 9=highest compression)
            outJar.setLevel(getJarCompressionLevel());

            // Create a read buffer
            byte[] inbuf = new byte[65000];

            // Write each temporary file to the Jar file
            for (int i = 0; i < saveReq.getNumberOfFiles(); i++) {

                // Get the current temporary file
                CachedFileInfo finfo = saveReq.getFileInfo(i);
                FileState fstate = finfo.getFileState();

                // DEBUG
                if (Debug.EnableInfo && hasDebug())
                    Debug.println("DBFileLoader storeMultipleFile() info=" + finfo + ", fstate=" + fstate);

                if (fstate != null && fstate.fileExists() == true) {

                    // Create a Jar entry for the temporary file and write the entry to the Jar file
                    String entryName = null;
                    if (finfo.getStreamId() > 0)
                        entryName = getTempFilePrefix() + finfo.getFileId() + "_" + finfo.getStreamId() + ".tmp";
                    else
                        entryName = getTempFilePrefix() + finfo.getFileId() + ".tmp";

                    // Open the temporary file
                    FileInputStream tempFile = null;

                    try {

                        tempFile = new FileInputStream(finfo.getTemporaryPath());

                        // Create the Jar file entry
                        JarEntry jarEntry = new JarEntry(entryName);
                        outJar.putNextEntry(jarEntry);

                        // Write the temporary file data to the Jar file
                        int rdlen = tempFile.read(inbuf);

                        while (rdlen > 0) {
                            outJar.write(inbuf, 0, rdlen);
                            rdlen = tempFile.read(inbuf);
                        }

                        // Close the temporary file, close the Jar file entry
                        tempFile.close();
                        outJar.closeEntry();
                    }
                    catch (Exception ex) {

                        // DEBUG
                        if (Debug.EnableError && hasDebug()) {
                            Debug.println("Failed to store " + finfo.getTemporaryPath());
                            Debug.println(ex);
                        }
                    }
                } else if (Debug.EnableInfo && hasDebug()) {

                    // DEBUG
                    Debug.println("## DBFileLoader storeMultipleFile() ignored file " + finfo.getTemporaryPath()
                            + ", exists=false");
                }
            }
        }
        catch (IOException ex) {
            Debug.println(ex);
        }
        finally {

            // Close the Jar file
            if (outJar != null) {
                try {
                    outJar.close();
                }
                catch (Exception ex) {
                }
            }
        }

        // Write the Jar file to the database
        int saveSts = StsRequeue;

        try {

            // Create a list of the files/streams contained in the Jar file
            DBDataDetailsList fileList = new DBDataDetailsList();

            for (int i = 0; i < saveReq.getNumberOfFiles(); i++) {

                // Get the current cached file
                CachedFileInfo finfo = saveReq.getFileInfo(i);

                // Add details of the file/stream to the Jar file list
                fileList.addFile(new DBDataDetails(finfo.getFileId(), finfo.getStreamId()));
            }

            // Save the Jar file data to the database
            getDBDataInterface().saveJarData(jarFile.getAbsolutePath(), fileList);

            // Indicate that the database update was successful
            saveSts = StsSuccess;

            // Delete the temporary Jar file
            if (hasKeepJars() == false)
                jarFile.delete();

            // Update the file segment state for all files in the transaction
            for (int i = 0; i < saveReq.getNumberOfFiles(); i++) {

                // Get the current cached file
                CachedFileInfo finfo = saveReq.getFileInfo(i);

                // Clear the cached file state
                if (finfo.hasFileState()) {
                    FileSegmentInfo fileSegInfo = (FileSegmentInfo) finfo.getFileState().findAttribute(DBFileSegmentInfo);
                    if (fileSegInfo != null) {
                        fileSegInfo.setQueued(false);
                        fileSegInfo.setUpdated(false);
                        fileSegInfo.setStatus(FileSegmentInfo.Saved);
                    }
                }
            }
        }
        catch (DBException ex) {
            Debug.println(ex);
        }
        catch (IOException ex) {
            Debug.println(ex);
        }

        // Return the data save status
        return saveSts;
    }

    /**
     * Initialize the file loader using the specified parameters
     *
     * @param params ConfigElement
     * @param ctx    DeviceContext
     * @throws FileLoaderException Failed to initialize the file loader
     * @throws IOException I/O error
     */
    public void initializeLoader(ConfigElement params, DeviceContext ctx)
            throws FileLoaderException, IOException {

        // Debug output enable
        if (params.getChild("Debug") != null)
            m_debug = true;

        // Get the count of worker threads to create
        ConfigElement nameVal = params.getChild("ThreadPoolSize");
        if (nameVal != null && (nameVal.getValue() == null || nameVal.getValue().length() == 0))
            throw new FileLoaderException("FileLoader ThreadPoolSize parameter is null");

        // Convert the thread pool size parameter, or use the default value
        m_readWorkers = DefaultWorkerThreads;
        m_writeWorkers = DefaultWorkerThreads;

        if (nameVal != null) {
            try {

                // Check for a single value or split read/write values
                String numVal = nameVal.getValue();
                int rdCnt = -1;
                int wrtCnt = -1;
                int pos = numVal.indexOf(':');

                if (pos == -1) {

                    // Use the same number of read and write worker threads
                    rdCnt = Integer.parseInt(numVal);
                    wrtCnt = rdCnt;
                } else {

                    // Split the string value into read and write values, and convert to integers
                    String val = numVal.substring(0, pos);
                    rdCnt = Integer.parseInt(val);

                    val = numVal.substring(pos + 1);
                    wrtCnt = Integer.parseInt(val);
                }

                // Set the read/write thread pool sizes
                m_readWorkers = rdCnt;
                m_writeWorkers = wrtCnt;
            }
            catch (NumberFormatException ex) {
                throw new FileLoaderException("DBFileLoader Invalid ThreadPoolSize value, " + ex.toString());
            }
        }

        // Range check the thread pool size
        if (m_readWorkers < MinimumWorkerThreads || m_readWorkers > MaximumWorkerThreads)
            throw new FileLoaderException("DBFileLoader Invalid ThreadPoolSize (read), valid range is " + MinimumWorkerThreads
                    + "-" + MaximumWorkerThreads);

        if (m_writeWorkers < MinimumWorkerThreads || m_writeWorkers > MaximumWorkerThreads)
            throw new FileLoaderException("DBFileLoader Invalid ThreadPoolSize (write), valid range is " + MinimumWorkerThreads
                    + "-" + MaximumWorkerThreads);

        // Get the temporary file data directory
        ConfigElement tempArea = params.getChild("TempDirectory");
        if (tempArea == null || tempArea.getValue() == null || tempArea.getValue().length() == 0)
            throw new FileLoaderException("FileLoader TempDirectory not specified or null");

        // Validate the temporary directory
        m_tempDirName = tempArea.getValue();
        if (m_tempDirName != null && m_tempDirName.endsWith(File.separator) == false)
            m_tempDirName = m_tempDirName + File.separator;

        m_tempDir = new File(m_tempDirName);
        if (m_tempDir.exists() == false || m_tempDir.isDirectory() == false)
            throw new FileLoaderException("FileLoader TempDirectory does not exist, or is not a directory, " + m_tempDirName);

        if (m_tempDir.canWrite() == false)
            throw new FileLoaderException("FileLoader TempDirectory is not writeable, " + m_tempDirName);

        // Create the starting temporary sub-directory
        createNewTempDirectory();

        // Check if the maxmimum files per sub-directory has been specified
        ConfigElement maxFiles = params.getChild("MaximumFilesPerDirectory");
        if (maxFiles != null) {
            try {
                m_tempMax = Integer.parseInt(maxFiles.getValue());

                // Range check the maximum files per sub-directory
                if (m_tempMax < 10 || m_tempMax > 20000)
                    throw new FileLoaderException("FileLoader MaximumFilesPerDirectory out of valid range (10-20000)");
            }
            catch (NumberFormatException ex) {
                throw new FileLoaderException("FileLoader MaximumFilesPerDirectory invalid, " + maxFiles.getValue());
            }
        } else
            m_tempMax = MaximumFilesPerSubDir;

        // Check if transaction support should be enabled. If enabled small files are bundled
        // together into a single
        // file request for special processing by the file loader storeFile() method.
        nameVal = params.getChild("SmallFileSize");

        if (nameVal != null) {

            // Use the default settings unless specified
            m_sizePerJar = DefaultSizePerJar;
            m_filesPerJar = DefaultFilesPerJar;

            // Parse/validate the small file size parameter
            try {

                // Convert the small file size
                m_smallFileSize = MemorySize.getByteValue(nameVal.getValue());

                // Range check the small file size
                if (m_smallFileSize < 0)
                    throw new FileLoaderException("Invalid small file size value, " + nameVal.getValue());
            }
            catch (NumberFormatException ex) {
                throw new FileLoaderException("Invalid small file size value, " + nameVal.getValue());
            }

            // Check if the files per Jar setting has been specified
            nameVal = params.getChild("FilesPerJar");

            if (nameVal != null) {
                try {

                    // Convert the files per Jar value
                    m_filesPerJar = Integer.parseInt(nameVal.getValue());

                    // Range check the files per Jar value
                    if (m_filesPerJar < MinimumFilesPerJar)
                        throw new FileLoaderException("Files per jar setting is below minimum of " + MinimumFilesPerJar);
                }
                catch (NumberFormatException ex) {
                    throw new FileLoaderException("Invalid files per Jar setting, " + nameVal.getValue());
                }
            }

            // Check if the size per Jar setting has been specified
            nameVal = params.getChild("SizePerJar");

            if (nameVal != null) {
                try {

                    // Convert the size per Jar value
                    m_sizePerJar = MemorySize.getByteValueInt(nameVal.getValue());

                    // Range check the size per Jar value
                    if (m_sizePerJar < MinimumSizePerJar)
                        throw new FileLoaderException("Size per jar setting is below minimum of " + MinimumSizePerJar);
                }
                catch (NumberFormatException ex) {
                    throw new FileLoaderException("Invalid size per Jar setting, " + nameVal.getValue());
                }
            }

            // Check if the transaction timeout setting has been specified
            m_tranTimeout = DefaultTransactionTimeout;

            nameVal = params.getChild("TransactionTimeout");

            if (nameVal != null) {
                try {

                    // Convert the transaction timeout value
                    m_tranTimeout = Long.parseLong(nameVal.getValue()) * 1000L;

                    // Range check the transaction timeout value
                    if (m_tranTimeout < MinimumTransactionTimeout || m_tranTimeout > MaximumTransactionTimeout)
                        throw new FileLoaderException("Invalid transaction timeout value, out of valid range, "
                                + nameVal.getValue());
                }
                catch (NumberFormatException ex) {
                    throw new FileLoaderException("Invalid transaction timeout value, " + nameVal.getValue());
                }
            }
        }

        // Check if there are any file processors configured
        ConfigElement fileProcs = params.getChild("FileProcessors");
        if (fileProcs != null && fileProcs.hasChildren()) {

            // Validate the file processor classes and add to the file loader
            List<ConfigElement> procList = fileProcs.getChildren();
            for (ConfigElement procElem : procList) {

                // Get the current file processor class name
                if (procElem.getValue() == null || procElem.getValue().length() == 0)
                    throw new FileLoaderException("Empty file processor class name");

                // Validate the file processor class name and create an instance of the file
                // processor
                try {

                    // Create the file processor instance
                    Object procObj = Class.forName(procElem.getValue()).newInstance();

                    // Check that it is a file processor implementation
                    if (procObj instanceof FileProcessor) {

                        // Add to the list of file processors
                        addFileProcessor((FileProcessor) procObj);
                    } else
                        throw new FileLoaderException("Class " + procElem.getValue() + " is not a FileProcessor implementation");
                }
                catch (ClassNotFoundException ex) {
                    throw new FileLoaderException("File processor class not found, " + procElem.getValue());
                }
                catch (InstantiationException ex) {
                    throw new FileLoaderException("File processor exception, " + ex.toString());
                }
                catch (IllegalAccessException ex) {
                    throw new FileLoaderException("File processor exception, " + ex.toString());
                }
            }
        }

        // Check if the fragment size has been specified
        ConfigElement nv = params.getChild("FragmentSize");

        if (nv != null) {

            // Set the file data fragment size
            m_fragSize = MemorySize.getByteValue(nv.getValue());

            // Range check the value
            if (m_fragSize < MIN_FRAGSIZE || m_fragSize > MAX_FRAGSIZE)
                throw new FileLoaderException("FragmentSize is out of valid range (64K - 20Mb");
        }

        // Check if transaction request Jar files should be kept in the temporary area
        nv = params.getChild("KeepJars");
        if (nv != null)
            m_keepJars = true;

        // Check if the Jar compression level has been specified
        m_jarCompressLevel = JarDefaultCompression;

        nv = params.getChild("JarCompressionLevel");
        if (nv != null) {
            try {

                // Convert the compression level value
                m_jarCompressLevel = Integer.parseInt(nv.getValue());

                // Check if the compression level is valid
                if (m_jarCompressLevel < 0 || m_jarCompressLevel > 9)
                    throw new FileLoaderException("Invalid Jar compression level, valid range is 0 - 9");
            }
            catch (NumberFormatException ex) {
                throw new FileLoaderException("Invalid Jar compression level, " + nv.getValue());
            }
        }

        // Check if the database interface being used supports the required features
        if (ctx instanceof DBDeviceContext) {

            // Access the database device context
            m_dbCtx = (DBDeviceContext) ctx;

            // Check if the request queue is supported by the database interface
            if (getContext().getDBInterface().supportsFeature(DBInterface.FeatureQueue) == false)
                throw new FileLoaderException("DBLoader requires queue support in database interface");

            if (getContext().getDBInterface() instanceof DBQueueInterface == false)
                throw new FileLoaderException("Database interface does not implement queue interface");

            // Check if the data store feature is supported by the database interface
            if (getContext().getDBInterface().supportsFeature(DBInterface.FeatureData) == false)
                throw new FileLoaderException("DBLoader requires data support in database interface");

            if (getContext().getDBInterface() instanceof DBDataInterface)
                m_dbDataInterface = (DBDataInterface) getContext().getDBInterface();
            else
                throw new FileLoaderException("Database interface does not implement data interface");

            // Check if the Jar data store feature is supported by the database interface, if Jar
            // files are enabled
            if (getSmallFileSize() > 0 && getContext().getDBInterface().supportsFeature(DBInterface.FeatureJarData) == false)
                throw new FileLoaderException("DBLoader requires Jar data support in database interface");
        } else
            throw new FileLoaderException("Requires database device context");

        // Check if background loader debug is enabled
        if (params.getChild("ThreadDebug") != null)
            m_threadDebug = true;

    }

    /**
     * Start the file loader
     *
     * @param ctx DeviceContext
     */
    public void startLoader(DeviceContext ctx) {

        // Get the file state cache from the context
        m_stateCache = getContext().getStateCache();

        // Add the file loader as a file state listener so that we can cleanup temporary data files
        m_stateCache.addStateListener(this);

        // Get the database interface
        DBQueueInterface dbQueue = null;

        if (getContext().getDBInterface() instanceof DBQueueInterface)
            dbQueue = (DBQueueInterface) getContext().getDBInterface();
        else
            throw new RuntimeException("Database interface does not implement queue interface");

        // Perform a queue cleanup before starting the thread pool. This will check the temporary
        // cache area and delete files that are not part of a queued save/transaction save request.
        FileRequestQueue recoveredQueue = null;

        try {

            // Cleanup the temporary cache area and queue
            recoveredQueue = dbQueue.performQueueCleanup(m_tempDir, TempDirPrefix, TempFilePrefix, JarFilePrefix);

            // DEBUG
            if (recoveredQueue != null && Debug.EnableInfo && hasDebug())
                Debug.println("[DBLoader] Cleanup recovered " + recoveredQueue.numberOfRequests() + " pending save files");
        }
        catch (DBException ex) {

            // DEBUG
            if (Debug.EnableError && hasDebug())
                Debug.println(ex);
        }

        // Check if there are any file save requests pending in the queue database
        FileRequestQueue saveQueue = new FileRequestQueue();

        try {
            dbQueue.loadFileRequests(1, FileRequest.SAVE, saveQueue, 1);
            dbQueue.loadFileRequests(1, FileRequest.TRANSSAVE, saveQueue, 1);
        }
        catch (DBException ex) {
        }

        // Create the background load/save thread pool
        m_backgroundLoader = new BackgroundLoadSave("DBLdr", dbQueue, m_stateCache, this);

        m_backgroundLoader.setReadWorkers(m_readWorkers);
        m_backgroundLoader.setWriteWorkers(m_writeWorkers);

        m_backgroundLoader.setDebug(m_threadDebug);

        // Start the worker thread pool
        m_backgroundLoader.startThreads(saveQueue.numberOfRequests());

        // Check if transactions are enabled, if so then start the transaction timer thread
        if (getSmallFileSize() > 0) {

            // Enable the transaction loader in the background loader
            m_backgroundLoader.enableTransactions();

            // Create the transaction timer thread to flush incomplete transaction requests
            m_transTimer = new TransactionTimer("DBLdrTransTimer", getTransactionTimeout());
        }
    }

    /**
     * Shutdown the file loader and release all resources
     *
     * @param immediate boolean
     */
    public void shutdownLoader(boolean immediate) {

        // Shutdown the background load/save thread pool
        if (m_backgroundLoader != null)
            m_backgroundLoader.shutdownThreads();

        // Shutdown the transaction timer thread, if active
        if (m_transTimer != null)
            m_transTimer.shutdownRequest();
    }

    /**
     * Run the file store file processors
     *
     * @param context DiskDeviceContext
     * @param state   FileState
     * @param segment FileSegment
     */
    protected final void runFileStoreProcessors(DiskDeviceContext context, FileState state, FileSegment segment) {

        // Check if there are any file processors configured
        if (m_fileProcessors == null || m_fileProcessors.numberOfProcessors() == 0)
            return;

        try {

            // Run all of the file store processors
            for (int i = 0; i < m_fileProcessors.numberOfProcessors(); i++) {

                // Get the current file processor
                FileProcessor fileProc = m_fileProcessors.getProcessorAt(i);

                // Run the file processor
                fileProc.processStoredFile(context, state, segment);
            }

            // Make sure the file segment is closed after processing
            segment.closeFile();
        }
        catch (Exception ex) {

            // DEBUG
            if (Debug.EnableError && hasDebug()) {
                Debug.println("$$ Store file processor exception");
                Debug.println(ex);
            }
        }
    }

    /**
     * Run the file load file processors
     *
     * @param context DiskDeviceContext
     * @param state   FileState
     * @param segment FileSegment
     */
    protected final void runFileLoadedProcessors(DiskDeviceContext context, FileState state, FileSegment segment) {

        // Check if there are any file processors configured
        if (m_fileProcessors == null || m_fileProcessors.numberOfProcessors() == 0)
            return;

        try {

            // Run all of the file load processors
            for (int i = 0; i < m_fileProcessors.numberOfProcessors(); i++) {

                // Get the current file processor
                FileProcessor fileProc = m_fileProcessors.getProcessorAt(i);

                // Run the file processor
                fileProc.processLoadedFile(context, state, segment);
            }

            // Make sure the file segment is closed after processing
            segment.closeFile();
        }
        catch (Exception ex) {

            // DEBUG
            if (Debug.EnableError && hasDebug()) {
                Debug.println("$$ Load file processor exception");
                Debug.println(ex);
            }
        }
    }

    /**
     * Re-create, or attach, a file request to the file state.
     *
     * @param fid      int
     * @param tempPath String
     * @param virtPath String
     * @param sts      int
     * @return FileState
     */
    protected final FileState createFileStateForRequest(int fid, String tempPath, String virtPath, int sts) {

        // Find, or create, the file state for the file/directory
        FileState state = m_stateCache.findFileState(virtPath, false);

        if (state == null) {

            // Create a new file state for the path
            state = m_stateCache.findFileState(virtPath, true);

            synchronized (state) {

                // Prevent the file state from expiring whilst the request is queued against it
                state.setExpiryTime(FileState.NoTimeout);

                // Indicate that the file exists, set the unique file id
                state.setFileStatus(FileStatus.FileExists);
                state.setFileId(fid);

                // Check if the file segment has been attached to the file state
                FileSegmentInfo fileSegInfo = (FileSegmentInfo) state.findAttribute(DBFileSegmentInfo);
                FileSegment fileSeg = null;

                if (fileSegInfo == null) {

                    // Create a new file segment
                    fileSegInfo = new FileSegmentInfo();
                    fileSegInfo.setTemporaryFile(tempPath);

                    fileSeg = new FileSegment(fileSegInfo, true);
                    fileSeg.setStatus(sts, true);

                    // Add the segment to the file state cache
                    state.addAttribute(DBFileSegmentInfo, fileSegInfo);
                } else {

                    // Make sure the file segment indicates its part of a queued request
                    fileSeg = new FileSegment(fileSegInfo, true);
                    fileSeg.setStatus(sts, true);
                }
            }
        }

        // Return the file state
        return state;
    }

    /**
     * Find the file segment for the specified virtual path
     *
     * @param virtPath String
     * @return FileSegment
     */
    protected final FileSegment findFileSegmentForPath(String virtPath) {

        // Get the file state for the virtual path
        FileState fstate = m_stateCache.findFileState(virtPath, false);
        if (fstate == null)
            return null;

        // Get the file segment
        FileSegmentInfo segInfo = null;
        FileSegment fileSeg = null;

        synchronized (fstate) {

            // Get the associated file segment
            segInfo = (FileSegmentInfo) fstate.findAttribute(DBFileSegmentInfo);
            fileSeg = new FileSegment(segInfo, true);
        }

        // Return the file segment
        return fileSeg;
    }

    /**
     * Determine if the loader supports NTFS streams
     *
     * @return boolean
     */
    public boolean supportsStreams() {

        // Check if the database implementation supports the NTFS streams feature
        if (getContext() != null)
            return getContext().getDBInterface().supportsFeature(DBInterface.FeatureNTFS);
        return true;
    }

    /**
     * Create a new temporary sub-directory
     */
    private final void createNewTempDirectory() {

        // Create the starting temporary sub-directory
        m_curTempName = m_tempDirName + getTempDirectoryPrefix() + m_curTempIdx++;
        m_curTempDir = new File(m_curTempName);

        if (m_curTempDir.exists() == false)
            m_curTempDir.mkdir();

        // Clear the temporary file count
        m_tempCount = 0;

        // DEBUG
        if (Debug.EnableInfo && hasDebug())
            Debug.println("DBFileLoader Created new temp directory - " + m_curTempName);
    }

    /**
     * File state has expired. The listener can control whether the file state is removed from the
     * cache, or not.
     *
     * @param state FileState
     * @return true to remove the file state from the cache, or false to leave the file state in the
     * cache
     */
    public boolean fileStateExpired(FileState state) {

        // Check if the file state has an associated file segment
        FileSegmentInfo segInfo = (FileSegmentInfo) state.findAttribute(DBFileSegmentInfo);
        boolean expire = true;

        if (segInfo != null) {

            // Check if the file has a request queued
            if (segInfo.isQueued() == false) {

                try {

                    // Delete the temporary file and reset the segment status so that the data may
                    // be loaded again if required.
                    if (segInfo.hasStatus() != FileSegmentInfo.Initial) {

                        // Delete the temporary file
                        try {
                            segInfo.deleteTemporaryFile();
                        }
                        catch (IOException ex) {

                            // DEBUG
                            if (Debug.EnableError) {
                                Debug.println("Delete temp file error: " + ex.toString());
                                File tempFile = new File(segInfo.getTemporaryFile());
                                Debug.println("  TempFile file=" + tempFile.getAbsolutePath() + ", exists=" + tempFile.exists());
                                Debug.println("  FileState state=" + state);
                                Debug.println("  FileSegmentInfo segInfo=" + segInfo);
                                Debug.println("  StateCache size=" + m_stateCache.numberOfStates());
                            }
                        }

                        // Remove the file segment, reset the file segment back to the initial state
                        state.removeAttribute(DBFileSegmentInfo);
                        segInfo.setStatus(FileSegmentInfo.Initial);

                        // Reset the file state to indicate file data load required
                        state.setDataStatus(FileState.DataStatus.LoadWait);

                        // Check if the temporary file sub-directory is now empty, and it is not the
                        // current temporary sub-directory
                        if (segInfo.getTemporaryFile().startsWith(m_curTempName) == false) {

                            // Check if the sub-directory is empty
                            File tempFile = new File(segInfo.getTemporaryFile());
                            File subDir = tempFile.getParentFile();
                            String[] files = subDir.list();
                            if (files == null || files.length == 0)
                                subDir.delete();
                        }

                        // Indicate that the file state should not be deleted
                        expire = false;

                        // Debug
                        if (Debug.EnableInfo && hasDebug())
                            Debug.println("$$ Deleted temporary file " + segInfo.getTemporaryFile() + " [EXPIRED] $$");
                    }

                    // If the file state is not to be deleted reset the file state expiration timer
                    if (expire == false)
                        state.setExpiryTime(System.currentTimeMillis() + m_stateCache.getFileStateExpireInterval());
                }
                catch (Exception ex) {

                    // DEBUG
                    if (Debug.EnableError) {
                        Debug.println("$$  " + ex.toString());
                        Debug.println("  state=" + state);
                    }
                }
            } else {

                // File state is queued, do not expire
                expire = false;
            }
        } else if (state.isDirectory()) {

            // Nothing to do when it's a directory, just allow it to expire
            expire = true;
        }

        // Return true if the file state can be expired
        return expire;
    }

    /**
     * File state cache is closing down, any resources attached to the file state must be released.
     *
     * @param state FileState
     */
    public void fileStateClosed(FileState state) {

        // DEBUG
        if (state == null) {
            Debug.println("%%%%% FileLoader.fileStateClosed() state=NULL %%%%%");
            return;
        }

        // Check if the file state has an associated file
        FileSegmentInfo segInfo = (FileSegmentInfo) state.findAttribute(DBFileSegmentInfo);

        if (segInfo != null && segInfo.isQueued() == false && segInfo.hasStatus() != FileSegmentInfo.SaveWait) {

            try {

                // Delete the temporary file
                segInfo.deleteTemporaryFile();

                // Debug
                if (Debug.EnableInfo && hasDebug())
                    Debug.println("$$ Deleted temporary file " + segInfo.getTemporaryFile() + " [CLOSED] $$");
            }
            catch (IOException ex) {
            }
        }
    }

    /**
     * Create a file segment to load/save the file data
     *
     * @param state  FileState
     * @param params FileOpenParams
     * @param fname  String
     * @param fid    int
     * @param stid   int
     * @param did    int
     * @return CachedNetworkFile
     * @throws IOException I/O error
     */
    private final CachedNetworkFile createNetworkFile(FileState state, FileOpenParams params, String fname, int fid, int stid,
                                                      int did)
            throws IOException {

        // The file state is used to synchronize the creation of the file segment as there may be
        // other sessions opening the file at the same time. We have to be careful that only one thread
        // creates the file segment.
        FileSegment fileSeg = null;
        CachedNetworkFile netFile = null;

        synchronized (state) {

            // Check if the file segment has been attached to the file state
            FileSegmentInfo fileSegInfo = (FileSegmentInfo) state.findAttribute(DBFileSegmentInfo);
            if (fileSegInfo == null) {

                // Check if we need to create a new temporary sub-drectory
                if (m_tempCount++ >= m_tempMax)
                    createNewTempDirectory();

                // Create a unique temporary file name
                StringBuffer tempName = new StringBuffer();
                tempName.append(getTempFilePrefix());
                tempName.append(fid);

                if (stid > 0) {
                    tempName.append("_");
                    tempName.append(stid);

                    // DEBUG
                    if (Debug.EnableInfo)
                        Debug.println("## Temp file for stream ##");
                }

                tempName.append(".tmp");

                // Create a new file segment
                fileSegInfo = new FileSegmentInfo();
                fileSeg = FileSegment.createSegment(fileSegInfo, tempName.toString(), m_curTempDir,
                        params.isReadOnlyAccess() == false);

                // Add the segment to the file state cache
                state.addAttribute(DBFileSegmentInfo, fileSegInfo);

                // Check if the file is zero length, if so then set the file segment state to
                // indicate it is available
                DBFileInfo finfo = (DBFileInfo) state.findAttribute(FileState.FileInformation);
                if (finfo != null && finfo.getSize() == 0)
                    fileSeg.setStatus(FileSegmentInfo.Available);
            } else {

                // Create the file segment to map to the existing temporary file
                fileSeg = new FileSegment(fileSegInfo, params.isReadOnlyAccess() == false);

                // Check if the temporary file exists, if not then create it
                File tempFile = new File(fileSeg.getTemporaryFile());
                if (tempFile.exists() == false) {

                    // Create the temporary file
                    tempFile.createNewFile();

                    // Reset the file segment state to indicate a file load is required
                    fileSeg.setStatus(FileSegmentInfo.Initial);
                }
            }

            // Create the new network file
            FileStateProxy stateProxy = m_stateCache.getFileStateProxy(state);
            netFile = new CachedNetworkFile(fname, fid, stid, did, stateProxy, fileSeg, this);

            netFile.setGrantedAccess(params.isReadOnlyAccess() ? NetworkFile.Access.READ_ONLY : NetworkFile.Access.READ_WRITE);
            netFile.setSequentialOnly(params.isSequentialAccessOnly());
            netFile.setAttributes(params.getAttributes());
            netFile.setFullName(params.getPath());

            if (stid != 0)
                netFile.setStreamName(params.getStreamName());
        }

        // Return the network file
        return netFile;
    }

    /**
     * Create a transaction file request
     *
     * @param req     SingleFileRequest
     * @param netFile NetworkFile
     */
    private final void createTransactionRequest(SingleFileRequest req, NetworkFile netFile) {

        synchronized (m_tranLock) {

            // Add the current file size to the current transaction size, and file count
            m_totFiles++;

            int fsize = netFile.getFileSizeInt();
            if (fsize < TransactionMinimumFileSize)
                fsize = TransactionMinimumFileSize;

            m_totFileSize += fsize;

            // Check if a new transaction should be started
            boolean lastFile = false;

            if (getFilesPerJar() > 0 && m_totFiles > getFilesPerJar())
                lastFile = true;
            else if (getJarFileSize() > 0 && m_totFileSize >= getJarFileSize())
                lastFile = true;

            // Set the transaction id for the request
            req.setTransactionId(m_tranId, lastFile);

            // Store the time this file was added to the transaction
            m_lastTranFile = System.currentTimeMillis();

            // Start a new transaction if this is the last file
            if (lastFile) {
                m_tranId++;
                m_totFiles = 0;
                m_totFileSize = 0;
                m_lastTranFile = 0L;
            }
        }
    }

    /**
     * Set the database context
     *
     * @param dbCtx DBDeviceContext
     */
    public final void setContext(DBDeviceContext dbCtx) {
        m_dbCtx = dbCtx;
    }
}
